{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 32645658.0\n",
      "1 27663066.0\n",
      "2 24636720.0\n",
      "3 20535462.0\n",
      "4 15307067.0\n",
      "5 10205002.0\n",
      "6 6345000.0\n",
      "7 3885080.25\n",
      "8 2460992.75\n",
      "9 1658451.0\n",
      "10 1195689.125\n",
      "11 914238.875\n",
      "12 730388.0\n",
      "13 601280.1875\n",
      "14 505080.9375\n",
      "15 430098.84375\n",
      "16 369952.25\n",
      "17 320555.4375\n",
      "18 279296.90625\n",
      "19 244553.859375\n",
      "20 214975.796875\n",
      "21 189608.59375\n",
      "22 167773.4375\n",
      "23 148898.6875\n",
      "24 132524.8125\n",
      "25 118244.5703125\n",
      "26 105774.328125\n",
      "27 94839.2109375\n",
      "28 85200.28125\n",
      "29 76682.390625\n",
      "30 69137.125\n",
      "31 62441.12109375\n",
      "32 56482.8671875\n",
      "33 51167.515625\n",
      "34 46416.40234375\n",
      "35 42160.93359375\n",
      "36 38341.36328125\n",
      "37 34907.5078125\n",
      "38 31816.5390625\n",
      "39 29031.4140625\n",
      "40 26520.771484375\n",
      "41 24253.166015625\n",
      "42 22203.0078125\n",
      "43 20344.482421875\n",
      "44 18657.544921875\n",
      "45 17126.03125\n",
      "46 15736.25\n",
      "47 14469.9580078125\n",
      "48 13315.4931640625\n",
      "49 12261.6875\n",
      "50 11298.8017578125\n",
      "51 10418.2763671875\n",
      "52 9612.587890625\n",
      "53 8876.0498046875\n",
      "54 8201.224609375\n",
      "55 7582.33203125\n",
      "56 7013.9248046875\n",
      "57 6491.9052734375\n",
      "58 6012.18408203125\n",
      "59 5570.7841796875\n",
      "60 5164.29443359375\n",
      "61 4789.736328125\n",
      "62 4444.34423828125\n",
      "63 4125.75244140625\n",
      "64 3831.7578125\n",
      "65 3560.240966796875\n",
      "66 3309.258544921875\n",
      "67 3077.2197265625\n",
      "68 2862.605224609375\n",
      "69 2663.908447265625\n",
      "70 2479.925048828125\n",
      "71 2309.470947265625\n",
      "72 2151.48046875\n",
      "73 2004.9578857421875\n",
      "74 1869.025146484375\n",
      "75 1742.839111328125\n",
      "76 1625.6700439453125\n",
      "77 1516.8309326171875\n",
      "78 1415.709228515625\n",
      "79 1321.6949462890625\n",
      "80 1234.2884521484375\n",
      "81 1152.951416015625\n",
      "82 1077.2606201171875\n",
      "83 1006.8023681640625\n",
      "84 941.1860961914062\n",
      "85 880.070068359375\n",
      "86 823.1184692382812\n",
      "87 770.0249633789062\n",
      "88 720.552001953125\n",
      "89 674.4150390625\n",
      "90 631.3821411132812\n",
      "91 591.2264404296875\n",
      "92 553.7435913085938\n",
      "93 518.7490844726562\n",
      "94 486.05914306640625\n",
      "95 455.5237121582031\n",
      "96 426.9936828613281\n",
      "97 400.33013916015625\n",
      "98 375.4188232421875\n",
      "99 352.1171875\n",
      "100 330.3228454589844\n",
      "101 309.93109130859375\n",
      "102 290.852294921875\n",
      "103 272.9930725097656\n",
      "104 256.2784118652344\n",
      "105 240.6263427734375\n",
      "106 225.963623046875\n",
      "107 212.22633361816406\n",
      "108 199.3546142578125\n",
      "109 187.3606414794922\n",
      "110 176.1235809326172\n",
      "111 165.59115600585938\n",
      "112 155.71530151367188\n",
      "113 146.45306396484375\n",
      "114 137.76405334472656\n",
      "115 129.6090087890625\n",
      "116 121.95886993408203\n",
      "117 114.7727279663086\n",
      "118 108.03021240234375\n",
      "119 101.6978530883789\n",
      "120 95.74920654296875\n",
      "121 90.16146087646484\n",
      "122 84.91288757324219\n",
      "123 79.97806549072266\n",
      "124 75.34143829345703\n",
      "125 70.9808120727539\n",
      "126 66.88025665283203\n",
      "127 63.02635955810547\n",
      "128 59.401859283447266\n",
      "129 55.99072265625\n",
      "130 52.783870697021484\n",
      "131 49.7648811340332\n",
      "132 46.92459487915039\n",
      "133 44.249977111816406\n",
      "134 41.73278045654297\n",
      "135 39.362361907958984\n",
      "136 37.13079833984375\n",
      "137 35.030296325683594\n",
      "138 33.05156326293945\n",
      "139 31.187206268310547\n",
      "140 29.430461883544922\n",
      "141 27.77689552307129\n",
      "142 26.217355728149414\n",
      "143 24.747905731201172\n",
      "144 23.36298179626465\n",
      "145 22.05791664123535\n",
      "146 20.827531814575195\n",
      "147 19.667110443115234\n",
      "148 18.572694778442383\n",
      "149 17.540987014770508\n",
      "150 16.56756591796875\n",
      "151 15.649957656860352\n",
      "152 14.784327507019043\n",
      "153 13.967608451843262\n",
      "154 13.196914672851562\n",
      "155 12.46960163116455\n",
      "156 11.783459663391113\n",
      "157 11.136001586914062\n",
      "158 10.524513244628906\n",
      "159 9.947233200073242\n",
      "160 9.402939796447754\n",
      "161 8.888317108154297\n",
      "162 8.403033256530762\n",
      "163 7.944757461547852\n",
      "164 7.511530876159668\n",
      "165 7.10245943069458\n",
      "166 6.716156482696533\n",
      "167 6.35098123550415\n",
      "168 6.006676197052002\n",
      "169 5.681159019470215\n",
      "170 5.373538970947266\n",
      "171 5.0831098556518555\n",
      "172 4.8085856437683105\n",
      "173 4.548969268798828\n",
      "174 4.303530216217041\n",
      "175 4.071645259857178\n",
      "176 3.8525352478027344\n",
      "177 3.6453988552093506\n",
      "178 3.449615955352783\n",
      "179 3.264397144317627\n",
      "180 3.089545965194702\n",
      "181 2.924222946166992\n",
      "182 2.7676587104797363\n",
      "183 2.6197474002838135\n",
      "184 2.4798896312713623\n",
      "185 2.3473570346832275\n",
      "186 2.2223763465881348\n",
      "187 2.1039977073669434\n",
      "188 1.9919710159301758\n",
      "189 1.8862580060958862\n",
      "190 1.7858599424362183\n",
      "191 1.6908937692642212\n",
      "192 1.6012598276138306\n",
      "193 1.5163058042526245\n",
      "194 1.4359664916992188\n",
      "195 1.3600118160247803\n",
      "196 1.2881965637207031\n",
      "197 1.2201584577560425\n",
      "198 1.1558001041412354\n",
      "199 1.094690203666687\n",
      "200 1.037055492401123\n",
      "201 0.9824360609054565\n",
      "202 0.9306735396385193\n",
      "203 0.8817567825317383\n",
      "204 0.8354056477546692\n",
      "205 0.7914701700210571\n",
      "206 0.7500084042549133\n",
      "207 0.7106344699859619\n",
      "208 0.6733778119087219\n",
      "209 0.6380454301834106\n",
      "210 0.6047414541244507\n",
      "211 0.5730196833610535\n",
      "212 0.5430569052696228\n",
      "213 0.5147414207458496\n",
      "214 0.4879187047481537\n",
      "215 0.46247056126594543\n",
      "216 0.4383045732975006\n",
      "217 0.41548362374305725\n",
      "218 0.3938586115837097\n",
      "219 0.3733730614185333\n",
      "220 0.3539111316204071\n",
      "221 0.3355407118797302\n",
      "222 0.3181264400482178\n",
      "223 0.30163654685020447\n",
      "224 0.2859989404678345\n",
      "225 0.2711740732192993\n",
      "226 0.2571040689945221\n",
      "227 0.2437441200017929\n",
      "228 0.23107531666755676\n",
      "229 0.21918565034866333\n",
      "230 0.20784907042980194\n",
      "231 0.19710414111614227\n",
      "232 0.18693363666534424\n",
      "233 0.17723460495471954\n",
      "234 0.1680874228477478\n",
      "235 0.15947787463665009\n",
      "236 0.15123958885669708\n",
      "237 0.1434653252363205\n",
      "238 0.1360764056444168\n",
      "239 0.12908701598644257\n",
      "240 0.12245350331068039\n",
      "241 0.11618582159280777\n",
      "242 0.11021032929420471\n",
      "243 0.10456160455942154\n",
      "244 0.0991799384355545\n",
      "245 0.09407708793878555\n",
      "246 0.08924213796854019\n",
      "247 0.0846807137131691\n",
      "248 0.08034924417734146\n",
      "249 0.07624118030071259\n",
      "250 0.07235554605722427\n",
      "251 0.06864170730113983\n",
      "252 0.06514791399240494\n",
      "253 0.06182456016540527\n",
      "254 0.05866452679038048\n",
      "255 0.055698368698358536\n",
      "256 0.05284527689218521\n",
      "257 0.0501437671482563\n",
      "258 0.047591328620910645\n",
      "259 0.04516376182436943\n",
      "260 0.04287276044487953\n",
      "261 0.04070828855037689\n",
      "262 0.038634829223155975\n",
      "263 0.036676075309515\n",
      "264 0.03480109944939613\n",
      "265 0.03304711729288101\n",
      "266 0.03138662502169609\n",
      "267 0.02979419380426407\n",
      "268 0.028280820697546005\n",
      "269 0.02685992419719696\n",
      "270 0.025494350120425224\n",
      "271 0.024212365970015526\n",
      "272 0.0229873675853014\n",
      "273 0.021829552948474884\n",
      "274 0.020723652094602585\n",
      "275 0.019686033949255943\n",
      "276 0.01868753880262375\n",
      "277 0.017760884016752243\n",
      "278 0.016862738877534866\n",
      "279 0.016026606783270836\n",
      "280 0.015229981392621994\n",
      "281 0.014466932974755764\n",
      "282 0.013750732876360416\n",
      "283 0.013059231452643871\n",
      "284 0.012408148497343063\n",
      "285 0.011798406951129436\n",
      "286 0.011212674900889397\n",
      "287 0.010657957755029202\n",
      "288 0.010126627050340176\n",
      "289 0.00963682122528553\n",
      "290 0.009150885976850986\n",
      "291 0.008715116418898106\n",
      "292 0.008287384174764156\n",
      "293 0.007878025993704796\n",
      "294 0.007495758589357138\n",
      "295 0.007131464313715696\n",
      "296 0.00679031852632761\n",
      "297 0.006461761891841888\n",
      "298 0.00614533293992281\n",
      "299 0.005857722833752632\n",
      "300 0.005574285984039307\n",
      "301 0.005308519583195448\n",
      "302 0.0050581335090100765\n",
      "303 0.004810809623450041\n",
      "304 0.004588597919791937\n",
      "305 0.004371730610728264\n",
      "306 0.004172741901129484\n",
      "307 0.003976505249738693\n",
      "308 0.0037940433248877525\n",
      "309 0.0036206285003572702\n",
      "310 0.0034512963611632586\n",
      "311 0.0032966032158583403\n",
      "312 0.0031427256762981415\n",
      "313 0.003001162549480796\n",
      "314 0.0028639493975788355\n",
      "315 0.0027365749701857567\n",
      "316 0.0026127537712454796\n",
      "317 0.00249906862154603\n",
      "318 0.0023889329750090837\n",
      "319 0.0022825479973107576\n",
      "320 0.00218613026663661\n",
      "321 0.0020870657172054052\n",
      "322 0.001996231498196721\n",
      "323 0.0019108111737295985\n",
      "324 0.0018307921709492803\n",
      "325 0.0017513517523184419\n",
      "326 0.0016790020745247602\n",
      "327 0.001611068844795227\n",
      "328 0.0015416196547448635\n",
      "329 0.0014772182330489159\n",
      "330 0.001417335937730968\n",
      "331 0.0013575098710134625\n",
      "332 0.0013021774357184768\n",
      "333 0.0012520074378699064\n",
      "334 0.0012022005394101143\n",
      "335 0.001154307508841157\n",
      "336 0.0011099646799266338\n",
      "337 0.0010670061456039548\n",
      "338 0.0010257296962663531\n",
      "339 0.0009880373254418373\n",
      "340 0.0009504095651209354\n",
      "341 0.0009149258839897811\n",
      "342 0.0008789976127445698\n",
      "343 0.0008465881110168993\n",
      "344 0.0008160454453900456\n",
      "345 0.0007872951682657003\n",
      "346 0.0007590583409182727\n",
      "347 0.0007321325247175992\n",
      "348 0.000704356818459928\n",
      "349 0.0006811560597270727\n",
      "350 0.0006561647751368582\n",
      "351 0.0006321402615867555\n",
      "352 0.0006104958010837436\n",
      "353 0.0005896863876841962\n",
      "354 0.0005689237732440233\n",
      "355 0.0005507721798494458\n",
      "356 0.0005325726815499365\n",
      "357 0.0005152352969162166\n",
      "358 0.0004972651368007064\n",
      "359 0.00048075037193484604\n",
      "360 0.0004665287269745022\n",
      "361 0.00045121691073291004\n",
      "362 0.0004374403797555715\n",
      "363 0.00042304088128730655\n",
      "364 0.00040940140024758875\n",
      "365 0.0003967422526329756\n",
      "366 0.00038512024912051857\n",
      "367 0.0003730124735739082\n",
      "368 0.00036185275530442595\n",
      "369 0.00035067222779616714\n",
      "370 0.0003411492216400802\n",
      "371 0.0003307170991320163\n",
      "372 0.0003220027720090002\n",
      "373 0.00031262225820682943\n",
      "374 0.00030286319088190794\n",
      "375 0.00029373326105996966\n",
      "376 0.0002857917861547321\n",
      "377 0.0002785047108773142\n",
      "378 0.0002708297106437385\n",
      "379 0.00026318227173760533\n",
      "380 0.0002561558794695884\n",
      "381 0.00024946266785264015\n",
      "382 0.0002420396776869893\n",
      "383 0.0002360812941333279\n",
      "384 0.00023038459767121822\n",
      "385 0.0002244229253847152\n",
      "386 0.00021827754972036928\n",
      "387 0.0002127874322468415\n",
      "388 0.00020761789346579462\n",
      "389 0.00020242070604581386\n",
      "390 0.00019724744197446853\n",
      "391 0.00019303837325423956\n",
      "392 0.0001882123906398192\n",
      "393 0.00018265614926349372\n",
      "394 0.00017847698472905904\n",
      "395 0.0001742699823807925\n",
      "396 0.00017000582010950893\n",
      "397 0.0001658334513194859\n",
      "398 0.00016221724217757583\n",
      "399 0.00015855877427384257\n",
      "400 0.00015422288561239839\n",
      "401 0.00015141502080950886\n",
      "402 0.00014833975001238286\n",
      "403 0.0001447966933483258\n",
      "404 0.00014163099694997072\n",
      "405 0.00013873599527869374\n",
      "406 0.00013550314179155976\n",
      "407 0.00013251112250145525\n",
      "408 0.00012948118092026561\n",
      "409 0.0001268645137315616\n",
      "410 0.00012414177763275802\n",
      "411 0.00012117958249291405\n",
      "412 0.00011853243631776422\n",
      "413 0.00011621342127909884\n",
      "414 0.00011382412776583806\n",
      "415 0.00011142834409838542\n",
      "416 0.00010962410306092352\n",
      "417 0.00010743708844529465\n",
      "418 0.00010524747631279752\n",
      "419 0.00010322703747078776\n",
      "420 0.00010085463873110712\n",
      "421 9.881702135317028e-05\n",
      "422 9.689680155133829e-05\n",
      "423 9.48966626310721e-05\n",
      "424 9.259783109882846e-05\n",
      "425 9.107586811296642e-05\n",
      "426 8.950755000114441e-05\n",
      "427 8.750944107305259e-05\n",
      "428 8.647358481539413e-05\n",
      "429 8.481442637275904e-05\n",
      "430 8.329858246725053e-05\n",
      "431 8.209430234273896e-05\n",
      "432 8.018087828531861e-05\n",
      "433 7.877674943301827e-05\n",
      "434 7.785967318341136e-05\n",
      "435 7.665381417609751e-05\n",
      "436 7.535839540651068e-05\n",
      "437 7.363657641690224e-05\n",
      "438 7.278636621776968e-05\n",
      "439 7.152135367505252e-05\n",
      "440 6.99562078807503e-05\n",
      "441 6.892740202601999e-05\n",
      "442 6.796367233619094e-05\n",
      "443 6.678916543023661e-05\n",
      "444 6.550053512910381e-05\n",
      "445 6.47116539767012e-05\n",
      "446 6.378249236149713e-05\n",
      "447 6.294916965998709e-05\n",
      "448 6.18065387243405e-05\n",
      "449 6.069209484849125e-05\n",
      "450 5.978827175567858e-05\n",
      "451 5.907860031584278e-05\n",
      "452 5.7991132052848116e-05\n",
      "453 5.7384433603147045e-05\n",
      "454 5.649707964039408e-05\n",
      "455 5.543292354559526e-05\n",
      "456 5.489174509420991e-05\n",
      "457 5.41522640560288e-05\n",
      "458 5.2811039495281875e-05\n",
      "459 5.20163775945548e-05\n",
      "460 5.1334314775886014e-05\n",
      "461 5.088365651317872e-05\n",
      "462 5.029783642385155e-05\n",
      "463 4.9454196414444596e-05\n",
      "464 4.8714013246353716e-05\n",
      "465 4.7970839659683406e-05\n",
      "466 4.715872637461871e-05\n",
      "467 4.671701753977686e-05\n",
      "468 4.6133292926242575e-05\n",
      "469 4.561531022773124e-05\n",
      "470 4.51102496299427e-05\n",
      "471 4.4478438212536275e-05\n",
      "472 4.369062662590295e-05\n",
      "473 4.3218082282692194e-05\n",
      "474 4.2665109504014254e-05\n",
      "475 4.207616439089179e-05\n",
      "476 4.151928806095384e-05\n",
      "477 4.09551321354229e-05\n",
      "478 4.0297927625942975e-05\n",
      "479 3.997167368652299e-05\n",
      "480 3.9432165067410097e-05\n",
      "481 3.8909776776563376e-05\n",
      "482 3.8511840102728456e-05\n",
      "483 3.8155885704327375e-05\n",
      "484 3.756584555958398e-05\n",
      "485 3.7146197428228334e-05\n",
      "486 3.679174915305339e-05\n",
      "487 3.601132266339846e-05\n",
      "488 3.5855777241522446e-05\n",
      "489 3.5401113564148545e-05\n",
      "490 3.4919543395517394e-05\n",
      "491 3.448366260272451e-05\n",
      "492 3.417353946133517e-05\n",
      "493 3.370946433278732e-05\n",
      "494 3.344992728671059e-05\n",
      "495 3.2925745472311974e-05\n",
      "496 3.2593241485301405e-05\n",
      "497 3.225218824809417e-05\n",
      "498 3.200711580575444e-05\n",
      "499 3.163660949212499e-05\n"
     ]
    }
   ],
   "source": [
    "device = torch.device('cpu')\n",
    "N, D_in, H, D_out = 64, 1000, 100, 10\n",
    "\n",
    "x = torch.randn(N, D_in, device=device)\n",
    "y = torch.randn(N, D_out, device=device)\n",
    "\n",
    "w1 = torch.randn(D_in, H, device=device, requires_grad=True)\n",
    "w2 = torch.randn(H, D_out, device=device, requires_grad=True)\n",
    "\n",
    "learning_rate = 1e-6\n",
    "for t in range(500):\n",
    "    y_pred = x.mm(w1).clamp(min=0).mm(w2)\n",
    "    loss = (y_pred - y).pow(2).sum()\n",
    "    print(t, loss.item())\n",
    "    loss.backward()\n",
    "    \n",
    "    with torch.no_grad():\n",
    "        w1 -= learning_rate * w1.grad\n",
    "        w2 -= learning_rate * w2.grad\n",
    "        \n",
    "        w1.grad.zero_()\n",
    "        w2.grad.zero_()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
